{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('/home/kazuki_onodera/Python')\n",
    "import lgbmextension as ex\n",
    "import lightgbm as lgb\n",
    "import multiprocessing\n",
    "from gplearn.genetic import SymbolicTransformer\n",
    "import gc\n",
    "import EDA\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SEED = 71"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:02<00:00,  7.43it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 35.06it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(307511, 120)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#X = utils.load_train().drop(['SK_ID_CURR'], axis=1)\n",
    "X = utils.read_pickles('../data/101_train')\n",
    "y = utils.read_pickles('../data/label').TARGET\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "         'objective': 'binary',\n",
    "         'metric': 'auc',\n",
    "         'learning_rate': 0.05,\n",
    "         'max_depth': -1,\n",
    "         'num_leaves': 255,\n",
    "         'max_bin': 255,\n",
    "         'colsample_bytree': 0.5,\n",
    "         'subsample': 0.5,\n",
    "         'nthread': multiprocessing.cpu_count(),\n",
    "         'bagging_freq': 1,\n",
    "         \n",
    "         'seed': SEED\n",
    "         }\n",
    "\n",
    "\n",
    "categorical_feature = ['NAME_CONTRACT_TYPE',\n",
    "                     'CODE_GENDER',\n",
    "                     'FLAG_OWN_CAR',\n",
    "                     'FLAG_OWN_REALTY',\n",
    "                     'NAME_TYPE_SUITE',\n",
    "                     'NAME_INCOME_TYPE',\n",
    "                     'NAME_EDUCATION_TYPE',\n",
    "                     'NAME_FAMILY_STATUS',\n",
    "                     'NAME_HOUSING_TYPE',\n",
    "                     'OCCUPATION_TYPE',\n",
    "                     'WEEKDAY_APPR_PROCESS_START',\n",
    "                     'ORGANIZATION_TYPE',\n",
    "                     'FONDKAPREMONT_MODE',\n",
    "                     'HOUSETYPE_MODE',\n",
    "                     'WALLSMATERIAL_MODE',\n",
    "                     'EMERGENCYSTATE_MODE']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CV ( benchmark )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10]\tcv_agg's auc: 0.729411 + 0.00215306\n",
      "[20]\tcv_agg's auc: 0.736183 + 0.00217633\n",
      "[30]\tcv_agg's auc: 0.741978 + 0.00206874\n",
      "[40]\tcv_agg's auc: 0.743821 + 0.0023523\n",
      "[50]\tcv_agg's auc: 0.746626 + 0.00228584\n",
      "[60]\tcv_agg's auc: 0.747173 + 0.00228664\n",
      "[70]\tcv_agg's auc: 0.747679 + 0.0020249\n",
      "[80]\tcv_agg's auc: 0.748786 + 0.00179863\n",
      "[90]\tcv_agg's auc: 0.749196 + 0.00171387\n",
      "[100]\tcv_agg's auc: 0.749337 + 0.00141182\n",
      "[110]\tcv_agg's auc: 0.74968 + 0.00179155\n",
      "[120]\tcv_agg's auc: 0.749964 + 0.00172182\n",
      "[130]\tcv_agg's auc: 0.750274 + 0.00157808\n",
      "[140]\tcv_agg's auc: 0.750476 + 0.00151443\n",
      "[150]\tcv_agg's auc: 0.750359 + 0.0015564\n",
      "[160]\tcv_agg's auc: 0.750387 + 0.00163539\n",
      "[170]\tcv_agg's auc: 0.750379 + 0.00169016\n",
      "[180]\tcv_agg's auc: 0.750083 + 0.00169213\n",
      "[190]\tcv_agg's auc: 0.749825 + 0.00167055\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7505534358850445"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtrain = lgb.Dataset(X, y, categorical_feature=categorical_feature)\n",
    "ret = lgb.cv(param, dtrain, 9999, nfold=5,\n",
    "             early_stopping_rounds=50, verbose_eval=10,\n",
    "             seed=SEED)\n",
    "ret['auc-mean'][-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CV ( encoding )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:03<00:00,  6.21it/s]\n"
     ]
    }
   ],
   "source": [
    "X = utils.load_train().drop(['SK_ID_CURR'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = X.drop('TARGET', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enc = ce.LeaveOneOutEncoder(cols=categorical_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tmp_ = enc.fit_transform(tmp, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (307511, 16)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DataType</th>\n",
       "      <th>#Nulls</th>\n",
       "      <th>#Uniques</th>\n",
       "      <th>Min</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Max</th>\n",
       "      <th>Std</th>\n",
       "      <th>top10 val</th>\n",
       "      <th>top10 cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.054783</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.083459</td>\n",
       "      <td>0.008417</td>\n",
       "      <td>[0.08345912763449208, 0.05478329177909082]</td>\n",
       "      <td>[278232, 29279]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.101419</td>\n",
       "      <td>0.014907</td>\n",
       "      <td>[0.06999328222555916, 0.1014192025433328, 0.0]</td>\n",
       "      <td>[202448, 105059, 4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.072437</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.085002</td>\n",
       "      <td>0.005953</td>\n",
       "      <td>[0.08500226685852831, 0.07243730100299273]</td>\n",
       "      <td>[202924, 104587]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.079616</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.083249</td>\n",
       "      <td>0.001675</td>\n",
       "      <td>[0.07961577407740775, 0.08324929139375152]</td>\n",
       "      <td>[213312, 94199]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.073768</td>\n",
       "      <td>0.080840</td>\n",
       "      <td>0.098305</td>\n",
       "      <td>0.002816</td>\n",
       "      <td>[0.08183047246565751, 0.07494582679518792, 0.0...</td>\n",
       "      <td>[248526, 40149, 11370, 3267, 1770, 1292, 866, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.017393</td>\n",
       "      <td>[0.09588471664126369, 0.07484256531270508, 0.0...</td>\n",
       "      <td>[158774, 71617, 55362, 21703, 28, 22, 5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.018293</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.109277</td>\n",
       "      <td>0.015683</td>\n",
       "      <td>[0.08939928843221562, 0.05355115344028425, 0.0...</td>\n",
       "      <td>[218391, 74863, 10277, 3816, 164]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.099446</td>\n",
       "      <td>0.011036</td>\n",
       "      <td>[0.07559868045939562, 0.0980767538068832, 0.09...</td>\n",
       "      <td>[196432, 45444, 29775, 19770, 16088, 2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.065724</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.123131</td>\n",
       "      <td>0.010074</td>\n",
       "      <td>[0.07795710746588094, 0.1169811320754717, 0.08...</td>\n",
       "      <td>[272868, 14840, 11183, 4881, 2617, 1122]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.048303</td>\n",
       "      <td>0.085618</td>\n",
       "      <td>0.171524</td>\n",
       "      <td>0.019403</td>\n",
       "      <td>[0.08072881945686496, 0.10578769977892943, 0.0...</td>\n",
       "      <td>[96391, 55186, 32102, 27570, 21371, 18603, 113...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.077572</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.083505</td>\n",
       "      <td>0.001927</td>\n",
       "      <td>[0.08350494424964286, 0.0816035737667039, 0.07...</td>\n",
       "      <td>[53901, 51934, 50714, 50591, 50338, 33852, 16181]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>57</td>\n",
       "      <td>0.031250</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.157540</td>\n",
       "      <td>0.019707</td>\n",
       "      <td>[0.09299623485115896, 0.05399646043269404, 0.1...</td>\n",
       "      <td>[67992, 55374, 38412, 16683, 11193, 10553, 104...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.058195</td>\n",
       "      <td>0.076995</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.005802</td>\n",
       "      <td>[0.08072881945686496, 0.06978193146417445, 0.0...</td>\n",
       "      <td>[210295, 73830, 12080, 5687, 5619]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.069434</td>\n",
       "      <td>0.075318</td>\n",
       "      <td>0.101401</td>\n",
       "      <td>0.005943</td>\n",
       "      <td>[0.08072881945686496, 0.06943383188374982, 0.1...</td>\n",
       "      <td>[154297, 150503, 1499, 1212]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0.047218</td>\n",
       "      <td>0.075364</td>\n",
       "      <td>0.096979</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>[0.08072881945686496, 0.06347668079951545, 0.0...</td>\n",
       "      <td>[156341, 66040, 64815, 9253, 5362, 2296, 1779,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EMERGENCYSTATE_MODE</th>\n",
       "      <td>float64</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.069649</td>\n",
       "      <td>0.075099</td>\n",
       "      <td>0.095790</td>\n",
       "      <td>0.005802</td>\n",
       "      <td>[0.06964899515768874, 0.08072881945686496, 0.0...</td>\n",
       "      <td>[159428, 145755, 2328]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           DataType  #Nulls  #Uniques       Min      Mean  \\\n",
       "NAME_CONTRACT_TYPE          float64       0         2  0.054783  0.080729   \n",
       "CODE_GENDER                 float64       0         3  0.000000  0.080729   \n",
       "FLAG_OWN_CAR                float64       0         2  0.072437  0.080729   \n",
       "FLAG_OWN_REALTY             float64       0         2  0.079616  0.080729   \n",
       "NAME_TYPE_SUITE             float64       0         8  0.073768  0.080840   \n",
       "NAME_INCOME_TYPE            float64       0         7  0.000000  0.080729   \n",
       "NAME_EDUCATION_TYPE         float64       0         5  0.018293  0.080729   \n",
       "NAME_FAMILY_STATUS          float64       0         6  0.000000  0.080729   \n",
       "NAME_HOUSING_TYPE           float64       0         6  0.065724  0.080729   \n",
       "OCCUPATION_TYPE             float64       0        19  0.048303  0.085618   \n",
       "WEEKDAY_APPR_PROCESS_START  float64       0         7  0.077572  0.080729   \n",
       "ORGANIZATION_TYPE           float64       0        57  0.031250  0.080729   \n",
       "FONDKAPREMONT_MODE          float64       0         5  0.058195  0.076995   \n",
       "HOUSETYPE_MODE              float64       0         4  0.069434  0.075318   \n",
       "WALLSMATERIAL_MODE          float64       0         8  0.047218  0.075364   \n",
       "EMERGENCYSTATE_MODE         float64       0         3  0.069649  0.075099   \n",
       "\n",
       "                                 Max       Std  \\\n",
       "NAME_CONTRACT_TYPE          0.083459  0.008417   \n",
       "CODE_GENDER                 0.101419  0.014907   \n",
       "FLAG_OWN_CAR                0.085002  0.005953   \n",
       "FLAG_OWN_REALTY             0.083249  0.001675   \n",
       "NAME_TYPE_SUITE             0.098305  0.002816   \n",
       "NAME_INCOME_TYPE            0.400000  0.017393   \n",
       "NAME_EDUCATION_TYPE         0.109277  0.015683   \n",
       "NAME_FAMILY_STATUS          0.099446  0.011036   \n",
       "NAME_HOUSING_TYPE           0.123131  0.010074   \n",
       "OCCUPATION_TYPE             0.171524  0.019403   \n",
       "WEEKDAY_APPR_PROCESS_START  0.083505  0.001927   \n",
       "ORGANIZATION_TYPE           0.157540  0.019707   \n",
       "FONDKAPREMONT_MODE          0.080729  0.005802   \n",
       "HOUSETYPE_MODE              0.101401  0.005943   \n",
       "WALLSMATERIAL_MODE          0.096979  0.007692   \n",
       "EMERGENCYSTATE_MODE         0.095790  0.005802   \n",
       "\n",
       "                                                                    top10 val  \\\n",
       "NAME_CONTRACT_TYPE                 [0.08345912763449208, 0.05478329177909082]   \n",
       "CODE_GENDER                    [0.06999328222555916, 0.1014192025433328, 0.0]   \n",
       "FLAG_OWN_CAR                       [0.08500226685852831, 0.07243730100299273]   \n",
       "FLAG_OWN_REALTY                    [0.07961577407740775, 0.08324929139375152]   \n",
       "NAME_TYPE_SUITE             [0.08183047246565751, 0.07494582679518792, 0.0...   \n",
       "NAME_INCOME_TYPE            [0.09588471664126369, 0.07484256531270508, 0.0...   \n",
       "NAME_EDUCATION_TYPE         [0.08939928843221562, 0.05355115344028425, 0.0...   \n",
       "NAME_FAMILY_STATUS          [0.07559868045939562, 0.0980767538068832, 0.09...   \n",
       "NAME_HOUSING_TYPE           [0.07795710746588094, 0.1169811320754717, 0.08...   \n",
       "OCCUPATION_TYPE             [0.08072881945686496, 0.10578769977892943, 0.0...   \n",
       "WEEKDAY_APPR_PROCESS_START  [0.08350494424964286, 0.0816035737667039, 0.07...   \n",
       "ORGANIZATION_TYPE           [0.09299623485115896, 0.05399646043269404, 0.1...   \n",
       "FONDKAPREMONT_MODE          [0.08072881945686496, 0.06978193146417445, 0.0...   \n",
       "HOUSETYPE_MODE              [0.08072881945686496, 0.06943383188374982, 0.1...   \n",
       "WALLSMATERIAL_MODE          [0.08072881945686496, 0.06347668079951545, 0.0...   \n",
       "EMERGENCYSTATE_MODE         [0.06964899515768874, 0.08072881945686496, 0.0...   \n",
       "\n",
       "                                                                    top10 cnt  \n",
       "NAME_CONTRACT_TYPE                                            [278232, 29279]  \n",
       "CODE_GENDER                                               [202448, 105059, 4]  \n",
       "FLAG_OWN_CAR                                                 [202924, 104587]  \n",
       "FLAG_OWN_REALTY                                               [213312, 94199]  \n",
       "NAME_TYPE_SUITE             [248526, 40149, 11370, 3267, 1770, 1292, 866, ...  \n",
       "NAME_INCOME_TYPE                     [158774, 71617, 55362, 21703, 28, 22, 5]  \n",
       "NAME_EDUCATION_TYPE                         [218391, 74863, 10277, 3816, 164]  \n",
       "NAME_FAMILY_STATUS                    [196432, 45444, 29775, 19770, 16088, 2]  \n",
       "NAME_HOUSING_TYPE                    [272868, 14840, 11183, 4881, 2617, 1122]  \n",
       "OCCUPATION_TYPE             [96391, 55186, 32102, 27570, 21371, 18603, 113...  \n",
       "WEEKDAY_APPR_PROCESS_START  [53901, 51934, 50714, 50591, 50338, 33852, 16181]  \n",
       "ORGANIZATION_TYPE           [67992, 55374, 38412, 16683, 11193, 10553, 104...  \n",
       "FONDKAPREMONT_MODE                         [210295, 73830, 12080, 5687, 5619]  \n",
       "HOUSETYPE_MODE                                   [154297, 150503, 1499, 1212]  \n",
       "WALLSMATERIAL_MODE          [156341, 66040, 64815, 9253, 5362, 2296, 1779,...  \n",
       "EMERGENCYSTATE_MODE                                    [159428, 145755, 2328]  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EDA.df_info(tmp_[categorical_feature])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ORGANIZATION_TYPE\n",
       "Advertising               0.081585\n",
       "Agriculture               0.104727\n",
       "Bank                      0.051855\n",
       "Business Entity Type 1    0.081384\n",
       "Business Entity Type 2    0.085284\n",
       "Business Entity Type 3    0.092996\n",
       "Cleaning                  0.111538\n",
       "Construction              0.116798\n",
       "Culture                   0.055409\n",
       "Electricity               0.066316\n",
       "Emergency                 0.071429\n",
       "Government                0.069781\n",
       "Hotel                     0.064182\n",
       "Housing                   0.079446\n",
       "Industry: type 1          0.110683\n",
       "Industry: type 10         0.064220\n",
       "Industry: type 11         0.086538\n",
       "Industry: type 12         0.037940\n",
       "Industry: type 13         0.134328\n",
       "Industry: type 2          0.072052\n",
       "Industry: type 3          0.106162\n",
       "Industry: type 4          0.101482\n",
       "Industry: type 5          0.068447\n",
       "Industry: type 6          0.071429\n",
       "Industry: type 7          0.080337\n",
       "Industry: type 8          0.125000\n",
       "Industry: type 9          0.066805\n",
       "Insurance                 0.056951\n",
       "Kindergarten              0.070349\n",
       "Legal Services            0.078689\n",
       "Medicine                  0.065845\n",
       "Military                  0.051253\n",
       "Mobile                    0.091483\n",
       "Other                     0.076425\n",
       "Police                    0.049979\n",
       "Postal                    0.084376\n",
       "Realtor                   0.106061\n",
       "Religion                  0.058824\n",
       "Restaurant                0.117062\n",
       "School                    0.059148\n",
       "Security                  0.099784\n",
       "Security Ministries       0.048632\n",
       "Self-employed             0.101739\n",
       "Services                  0.066032\n",
       "Telecom                   0.076256\n",
       "Trade: type 1             0.089080\n",
       "Trade: type 2             0.070000\n",
       "Trade: type 3             0.103379\n",
       "Trade: type 4             0.031250\n",
       "Trade: type 5             0.061224\n",
       "Trade: type 6             0.045959\n",
       "Trade: type 7             0.094496\n",
       "Transport: type 1         0.044776\n",
       "Transport: type 2         0.078040\n",
       "Transport: type 3         0.157540\n",
       "Transport: type 4         0.092812\n",
       "University                0.048983\n",
       "XNA                       0.053996\n",
       "Name: TARGET, dtype: float64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_.groupby('ORGANIZATION_TYPE').TARGET.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "DataFrame.dtypes for data must be int, float or bool. Did not expect the data types in fields NAME_CONTRACT_TYPE, CODE_GENDER, FLAG_OWN_CAR, FLAG_OWN_REALTY, NAME_TYPE_SUITE, NAME_INCOME_TYPE, NAME_EDUCATION_TYPE, NAME_FAMILY_STATUS, NAME_HOUSING_TYPE, OCCUPATION_TYPE, WEEKDAY_APPR_PROCESS_START, ORGANIZATION_TYPE, FONDKAPREMONT_MODE, HOUSETYPE_MODE, WALLSMATERIAL_MODE, EMERGENCYSTATE_MODE",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-fedda0f7a959>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m ret = lgb.cv(param, dtrain, 9999, nfold=5,\n\u001b[1;32m      5\u001b[0m              \u001b[0mearly_stopping_rounds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m              seed=SEED)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/lightgbm/engine.py\u001b[0m in \u001b[0;36mcv\u001b[0;34m(params, train_set, num_boost_round, folds, nfold, stratified, shuffle, metrics, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, fpreproc, verbose_eval, show_stdv, seed, callbacks)\u001b[0m\n\u001b[1;32m    416\u001b[0m     cvfolds = _make_n_folds(train_set, folds=folds, nfold=nfold,\n\u001b[1;32m    417\u001b[0m                             \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfpreproc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfpreproc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 418\u001b[0;31m                             stratified=stratified, shuffle=shuffle)\n\u001b[0m\u001b[1;32m    419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m     \u001b[0;31m# setup callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/lightgbm/engine.py\u001b[0m in \u001b[0;36m_make_n_folds\u001b[0;34m(full_data, folds, nfold, params, seed, fpreproc, stratified, shuffle)\u001b[0m\n\u001b[1;32m    253\u001b[0m     \u001b[0mMake\u001b[0m \u001b[0man\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mfold\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mof\u001b[0m \u001b[0mBooster\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mrandom\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    254\u001b[0m     \"\"\"\n\u001b[0;32m--> 255\u001b[0;31m     \u001b[0mfull_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfull_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m     \u001b[0mnum_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfull_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfolds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36mconstruct\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    855\u001b[0m                                 \u001b[0mweight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minit_score\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_score\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    856\u001b[0m                                 \u001b[0mpredictor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predictor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msilent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msilent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 857\u001b[0;31m                                 categorical_feature=self.categorical_feature, params=self.params)\n\u001b[0m\u001b[1;32m    858\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfree_raw_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    859\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m_lazy_init\u001b[0;34m(self, data, label, reference, weight, group, init_score, predictor, silent, feature_name, categorical_feature, params)\u001b[0m\n\u001b[1;32m    647\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas_categorical\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreference\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas_categorical\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m             \u001b[0mcategorical_feature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreference\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 649\u001b[0;31m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategorical_feature\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas_categorical\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_data_from_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategorical_feature\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpandas_categorical\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    650\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_label_from_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    651\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_has_header\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m_data_from_pandas\u001b[0;34m(data, feature_name, categorical_feature, pandas_categorical)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    269\u001b[0m             \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\"DataFrame.dtypes for data must be int, float or bool. Did not expect the data types in fields \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 270\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m', '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbad_fields\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    271\u001b[0m         \u001b[0;31m#data = data.values.astype('float')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: DataFrame.dtypes for data must be int, float or bool. Did not expect the data types in fields NAME_CONTRACT_TYPE, CODE_GENDER, FLAG_OWN_CAR, FLAG_OWN_REALTY, NAME_TYPE_SUITE, NAME_INCOME_TYPE, NAME_EDUCATION_TYPE, NAME_FAMILY_STATUS, NAME_HOUSING_TYPE, OCCUPATION_TYPE, WEEKDAY_APPR_PROCESS_START, ORGANIZATION_TYPE, FONDKAPREMONT_MODE, HOUSETYPE_MODE, WALLSMATERIAL_MODE, EMERGENCYSTATE_MODE"
     ]
    }
   ],
   "source": [
    "dtrain = lgb.Dataset(tmp_, y, \n",
    "#                     categorical_feature=categorical_feature\n",
    "                    )\n",
    "ret = lgb.cv(param, dtrain, 9999, nfold=5,\n",
    "             early_stopping_rounds=50, verbose_eval=10,\n",
    "             seed=SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 45.36it/s]\n"
     ]
    }
   ],
   "source": [
    "X = utils.load_train().set_index('SK_ID_CURR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:04<00:00,  4.84it/s]\n"
     ]
    }
   ],
   "source": [
    "prev = utils.read_pickles('../data/previous_application')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prev = prev.sort_values(['SK_ID_CURR', 'DAYS_LAST_DUE_1ST_VERSION'], ascending=[True, False]).drop_duplicates('SK_ID_CURR', keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     68952\n",
       "2     42169\n",
       "3     26905\n",
       "4     18496\n",
       "5     12364\n",
       "6      8587\n",
       "7      6262\n",
       "8      4449\n",
       "9      3161\n",
       "10     2408\n",
       "11     1777\n",
       "12     1372\n",
       "13     1052\n",
       "14      746\n",
       "15      631\n",
       "16      466\n",
       "17      360\n",
       "18      313\n",
       "19      231\n",
       "20      176\n",
       "21      159\n",
       "22      122\n",
       "23       98\n",
       "24       96\n",
       "25       71\n",
       "26       60\n",
       "27       57\n",
       "28       33\n",
       "29       27\n",
       "30       24\n",
       "31       19\n",
       "33       14\n",
       "32       13\n",
       "34       13\n",
       "36        9\n",
       "37        9\n",
       "35        8\n",
       "39        7\n",
       "38        6\n",
       "41        4\n",
       "43        4\n",
       "44        4\n",
       "52        3\n",
       "60        2\n",
       "46        2\n",
       "54        2\n",
       "42        2\n",
       "55        2\n",
       "62        1\n",
       "58        1\n",
       "56        1\n",
       "65        1\n",
       "67        1\n",
       "45        1\n",
       "51        1\n",
       "49        1\n",
       "48        1\n",
       "47        1\n",
       "40        1\n",
       "70        1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev.groupby('SK_ID_CURR').size().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev = prev.drop('SK_ID_PREV', axis=1).set_index('SK_ID_CURR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tmp = pd.concat([X, prev.add_prefix('prev_')], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = [c for c in tmp.columns if tmp[c].dtype=='O']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "le = LabelEncoder()\n",
    "for c in cat:\n",
    "    tmp[c].fillna('na dayo', inplace=True)\n",
    "    tmp[c] = le.fit_transform(tmp[c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = tmp.TARGET\n",
    "tmp.drop('TARGET', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtrain = lgb.Dataset(tmp.iloc[0:,0:20], y, \n",
    "                     #categorical_feature=cat\n",
    "                    )\n",
    "ret = lgb.cv(param, dtrain, 9999, nfold=5,\n",
    "             early_stopping_rounds=50, verbose_eval=10,\n",
    "             seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NAME_CONTRACT_TYPE',\n",
       " 'CODE_GENDER',\n",
       " 'FLAG_OWN_CAR',\n",
       " 'FLAG_OWN_REALTY',\n",
       " 'NAME_TYPE_SUITE',\n",
       " 'NAME_INCOME_TYPE',\n",
       " 'NAME_EDUCATION_TYPE',\n",
       " 'NAME_FAMILY_STATUS',\n",
       " 'NAME_HOUSING_TYPE',\n",
       " 'OCCUPATION_TYPE',\n",
       " 'WEEKDAY_APPR_PROCESS_START',\n",
       " 'ORGANIZATION_TYPE',\n",
       " 'FONDKAPREMONT_MODE',\n",
       " 'HOUSETYPE_MODE',\n",
       " 'WALLSMATERIAL_MODE',\n",
       " 'EMERGENCYSTATE_MODE',\n",
       " 'prev_NAME_CONTRACT_TYPE',\n",
       " 'prev_WEEKDAY_APPR_PROCESS_START',\n",
       " 'prev_NAME_CASH_LOAN_PURPOSE',\n",
       " 'prev_NAME_CONTRACT_STATUS',\n",
       " 'prev_NAME_PAYMENT_TYPE',\n",
       " 'prev_CODE_REJECT_REASON',\n",
       " 'prev_NAME_TYPE_SUITE',\n",
       " 'prev_NAME_CLIENT_TYPE',\n",
       " 'prev_NAME_GOODS_CATEGORY',\n",
       " 'prev_NAME_PORTFOLIO',\n",
       " 'prev_NAME_PRODUCT_TYPE',\n",
       " 'prev_CHANNEL_TYPE',\n",
       " 'prev_NAME_SELLER_INDUSTRY',\n",
       " 'prev_NAME_YIELD_GROUP',\n",
       " 'prev_PRODUCT_COMBINATION']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117000000.0"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp.max().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'prev' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-889e0a424e47>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprev\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'prev' is not defined"
     ]
    }
   ],
   "source": [
    "prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
